import configparser
import os
import sys
import time
import traceback

import pymysql
import requests
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
sPath = BaseDir.get_new_path(TopPath, "download", "zswj", 'download', "Detail")
sGUrl = r'http://www.libvideo.com/danpian/danpian.aspx?id='
HEADERS = {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive',
           'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)', }
sSqlList = list()
nCount = 0
StartTime = time.time()


# Proxies = {
#     'http': '210.42.41.120:8002',
#     'https': '210.42.41.120:8002'
# }


# 测试文字输出到文本
def output(value, files='log.txt'):
    f = open(files, 'w', encoding='utf-8')
    f.write(value)
    f.close()


# 数据库链接
def MajorDbConnect():
    cf = configparser.ConfigParser()
    try:
        cf.read(os.path.join(curPath, 'db.ini'))
    except:
        print('读取配置失败:' + traceback.format_exc())
        return None
    if cf:
        db_host = cf.get('db', 'host')
        db_user = cf.get('db', 'user')
        db_pw = cf.get('db', 'passwd')
        db_name = cf.get('db', 'db')
        db_charset = cf.get('db', 'chartset')
        conn = pymysql.connect(host=db_host, user=db_user, passwd=db_pw, db=db_name, charset=db_charset)
        if conn:
            return conn


# 获取详情页列表
def GetHtml(nNum):
    conn = MajorDbConnect()
    sMsg = None
    taskList = list()
    if conn:
        sSql = "SELECT vid FROM article WHERE stat = 0 AND failcount < 3  LIMIT {nNum}".format(nNum=nNum)
        cur = conn.cursor()
        try:
            cur.execute(sSql)
        except:
            print('* errSql:' + sSql)
            sMsg = '* errSynax:' + traceback.format_exc()

        taskList = cur.fetchall()
        cur.close()
        conn.close()

    return taskList


# 下载详情页面
def DownPage(taskList):
    global sSqlList, nCount
    vid = None
    for task in taskList:
        if not task:
            print('* Task Error')
            sys.exit(-1)

        vid = task[0]

        bFlag = True
        pageUrl = sGUrl + vid
        r = None
        print('开始下载:', pageUrl)
        (bFlag, r) = RequestDetailPage(pageUrl)

        # 判断是否请求详情页成功 bFlag（1：成功，0：失败）
        sSql = ''
        if not bFlag:
            sSql = "UPDATE article set failcount = failcount + 1 where vid = '{vid}'".format(vid=vid)
        else:
            html = r.content.decode('utf-8').strip()
            line = html.replace('\0', ' ').replace('\r', ' ').replace('\n', ' ')
            if not BaseDir.is_dir_exists(sPath):
                BaseDir.create_dir(sPath)
            outPathFile = os.path.join(sPath, vid + '.htm')
            with open(outPathFile, mode='w', encoding='utf-8') as f:
                f.write(line)
                nCount += 1
                print('写入文件数量：%d' % nCount)
                print('耗费时间: %d' % int(time.time() - StartTime) + '秒')

            sSql = "UPDATE article set stat = 1 where vid = '{vid}'".format(vid=vid)
            print('DOWN:' + str(nCount))

        sSqlList.append(sSql)
        if len(sSqlList) > 100:
            UpdateDbById()

    UpdateDbById()


# 请求详情页
def RequestDetailPage(pageUrl):
    sExMsg = None
    r = None
    try:
        r = requests.get(pageUrl, headers=HEADERS, timeout=20)
    except:
        sExMsg = '* exMsg:' + traceback.format_exc()
    finally:
        if 'r' in locals() and r:
            r.close()
    if sExMsg:
        print(sExMsg)
        return False, r

    if r.status_code != requests.codes.ok:
        print('error-url:' + r.url)
        print('r.status_code' + str(r.status_code))
        return False, r

    html = r.text.strip()
    if not html.find(r'<div class="topic">'):
        print('not found topic:' + r.url)
        return False, r

    if not html.endswith('</html>'):
        print('not endswith </html>')
        return False, r

    return True, r


# 更新数据库状态
def UpdateDbById():
    global sSqlList
    conn = MajorDbConnect()
    cur = conn.cursor()
    dbMsg = None
    if conn:
        for sql in sSqlList:
            try:
                cur.execute(sql)
            except:
                print('*errSql:' + sql)
                dbMsg = '*updateError:' + traceback.format_exc()
            if dbMsg:
                continue
        conn.commit()
        conn.close()
        sSqlList = list()


# ----------------------------------------------逻辑部分-----------------------------------------------
# ----------------------------------------------下载详情页---------------------------------------------
def main():
    # 开始作业
    while True:
        taskList = GetHtml(1000)
        if not taskList:
            break
        DownPage(taskList)


if __name__ == '__main__':
    main()
